package com.spider;

import com.alibaba.excel.EasyExcel;
import com.alibaba.excel.ExcelWriter;
import com.alibaba.excel.write.metadata.WriteSheet;
import com.lx.algorithm.code.Code11;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import javax.sound.midi.Track;
import javax.swing.plaf.LabelUI;
import java.io.ByteArrayOutputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.sql.Array;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;


import java.io.File;

import java.net.HttpURLConnection;
import java.net.URL;

/**
 * Description:
 * Copyright:   Copyright (c)2019
 * Company:     zefu
 *
 * @author: 张李鑫
 * @version: 1.0
 * Create at:   2022-01-02 17:18:22
 * <p>
 * Modification History:
 * Date         Author      Version     Description
 * ------------------------------------------------------------------
 * 2022-01-02     张李鑫                     1.0         1.0 Version
 */
public class Spider {


    //定义路径
    static String baseurl = "http://www.netbian.com/";
    static String geturl = "https://gt4.topease.net/Import/GetList?TagId=220914&Import_CountryId=-10&Import_CountryName=%E5%85%A8%E7%90%83&isKeywordSearch=&sort_import=&trade_date=0&ydpage=&count_import_num=9044&Import_CompanyName=&Import_Keyword=&isRecommend=on&sourceid_importer=0&input_num=1&pagenum=1&recordCount=9044&defaultImportRecordcount=8947&default_temp_delids=&bill_temp_delids=&billImportRecordcount=96&recommandImportRecordcount=0&recommandImportStdId=-1&paginationid=pagination_import&pagesize=20&_=1641115493395";
    static String filepath = "D:\\idea\\zlx-study-home";

    public static void main(String[] args) {
        String fileName = "C:\\Users\\zlx\\Desktop\\Excel.xlsx";
        ExcelWriter excelWriter = null;
        excelWriter = EasyExcel.write(fileName, DemoData.class).build();
        WriteSheet writeSheet = EasyExcel.writerSheet("模板").build();




        try {
            //453
            for (int j =0; j < 453; j++) {

                String geturl = "https://gt4.topease.net/Import/GetList?TagId=220914&Import_CountryId=-10&Import_CountryName=%E5%85%A8%E7%90%83&isKeywordSearch=&sort_import=&trade_date=0&ydpage=&count_import_num=9044&Import_CompanyName=&Import_Keyword=&isRecommend=on&sourceid_importer=0&input_num=" + j + "&pagenum=" + j + "&recordCount=9044&defaultImportRecordcount=8947&default_temp_delids=&bill_temp_delids=&billImportRecordcount=96&recommandImportRecordcount=0&recommandImportStdId=-1&paginationid=pagination_import&pagesize=20&_=1641115493395";
                List<String> srclists = getImgSrcListFromHtml(getHtml(geturl));

                if (srclists.size() == 0) {
                    continue;
                }
                ArrayList<DemoData> demoData1 = new ArrayList<>();
                for (int i1 = 0; i1 < srclists.size(); i1++) {
                    String html = getHtml(srclists.get(i1));
                    List<String> list = getImgSrcListFromHtmlI(html);
                    if (list.size()!=9){
                        continue;
                    }
                    DemoData demoData = new DemoData();
                    demoData.setId(list.get(0));
                    demoData.setName(list.get(1));
                    demoData.setUser(list.get(2));
                    demoData.setForm(list.get(3));
                    demoData.setPhone(list.get(4));
                    demoData.setTel(list.get(5));
                    demoData.setEmail(list.get(6));
                    demoData.setNet(list.get(7));
                    demoData.setDz(list.get(8));
                    demoData1.add(demoData);
                }
                excelWriter.write(demoData1, writeSheet);
                Excel.simpleWrite(demoData1);
                System.out.println(j);
            }
        } catch (Exception e) {
            e.printStackTrace();
        } finally {

            if (excelWriter != null) {
                excelWriter.finish();
            }

        }
    }

    ////        String html = getHtml("https://gt4.topease.net//Import/Index?token=5D968A302A74C728FD9F704F46E9339E24324C4A09AE15C718598B98D0633C752458CB0C64352E3674F8DCC2B9C33E764C5EB5BCB80160DF2E547612C7C67029AD539FCB9EF4D5DCBDD8C352C603C3185C59A660184BE7CB82B9F477D5516D00A90004CE33604C9D");
    ////        List<String> list = getImgSrcListFromHtmlI(html);
    ////        DemoData demoData = new DemoData();
    ////        demoData.setId(list.get(0));
    ////        demoData.setName(list.get(1));
    ////        demoData.setUser(list.get(2));
    ////        demoData.setForm(list.get(3));
    ////        demoData.setPhone(list.get(4));
    ////        demoData.setTel(list.get(5));
    ////        demoData.setEmail(list.get(6));
    ////        demoData.setNet(list.get(7));
    ////        demoData.setDz(list.get(8));
    ////        ArrayList<DemoData> demoData1 = new ArrayList<>();
    ////        demoData1.add(demoData);
    ////        System.out.println("下载完毕");

    /**
     * @param @param  html  页面内容
     * @param @return 图片路径数组
     * @return ArrayList<String>    返回类型
     * @throws
     * @Title: getImgSrcListFromHtml
     * @Description: 获取页面内容图片路径
     */
    public static List<String> getImgSrcListFromHtmlI(String html) {


        List<String> list = new ArrayList<>();
        //解析成html页面
        Document document = Jsoup.parse(html);
        //获取目标
//        Elements elements = document.select("div[class=content]").select("div[class=mainbar]")
//                .select("div[class=matter]").select("div[class=row]").select("div[class=col-md-12]").select("div[class=widget]").select("div[class=widget-content]")
//                .select("div[class=tab-content]")
//                .select("div[class=tab_content]").select("div[id=GetCompanyContact]").select("div[class=tab1]").select("table")
//                .select("tbody").select("tr");

        Elements elements = document.select("div[class=tab1]").select("table")
                .select("tbody").select("tr");

        int len = elements.size();
        for (int i = 0; i < len; i++) {
            Elements td = elements.get(i).select("td");
            int size = td.size();
            for (int i1 = 0; i1 < size; i1++) {
                list.add(td.get(i1).text());
            }
        }

        return list;
    }

    /**
     * @param @param  url
     * @param @return 页面内容
     * @return String  返回类型
     * @throws
     * @Title: getHtml
     * @Description: 获取页面内容
     */
    public static String getHtml(String url) {

        String html = null;
        try {
                                                            //UM_distinctid=17e1a0479d0386-052ba0d74fd051-4303066-510000-17e1a0479d179a; ASP.NET_SessionId=n4syh40ep3dbdtetpnrunbkf
            html = Jsoup.connect(url).header("cookie", "UM_distinctid=17e1a0479d0386-052ba0d74fd051-4303066-510000-17e1a0479d179a; ASP.NET_SessionId=n4syh40ep3dbdtetpnrunbkf").header("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36").header("Content-Type", " text/html; charset=utf-8").get().html();
        } catch (IOException e) {
            e.printStackTrace();
        }

        return html;
    }

    /**
     * @param @param  html  页面内容
     * @param @return 图片路径数组
     * @return ArrayList<String>    返回类型
     * @throws
     * @Title: getImgSrcListFromHtml
     * @Description: 获取页面内容图片路径
     */
    public static List<String> getImgSrcListFromHtml(String html) {
        List<String> list = new ArrayList<>();
        //解析成html页面
        Document document = Jsoup.parse(html);
        //获取目标
        Elements elements = document.select("dl[class=companylist]").select("dd");
        //.select("dl [class=companylist]").select("dd");
        int len = elements.size();
        for (int i = 0; i < len; i++) {
            String attr = elements.get(i).select("div[class=left]").select("p").select("a").get(0).attr("href");
            list.add("https://gt4.topease.net/" + attr);
        }
        return list;
    }

    /**
     * @param 页面内容
     * @return List<String>  返回页面url数组
     * @throws
     * @Title: getNextPage
     * @Description: 从页面内容中获取下一个页面路径
     */
    public static List<String> getNextPageUrl(String html) {
        List<String> list = new ArrayList<>();
        //解析成html页面
        Document document = Jsoup.parse(html);
        //获取目标
        Elements elements = document.select("div [class=list]").select("a");
        for (int i = 0; i < elements.size(); i++) {
            String url = baseurl + elements.get(i).attr("href");
            list.add(url);
        }
        return list;
    }

    /**
     * @param @param list 图片路径数组
     * @param @param filepath  保存文件夹位置
     * @return void    返回类型
     * @throws
     * @Title: downloadImg
     * @Description: 下载图片 -- 通过获取的流转成byte[]数组，再通过FileOutputStream写出
     */
    public static void downloadImg(List<String> list, String filepath) {
        URL newUrl = null;
        HttpURLConnection hconnection = null;
        InputStream inputStream = null;
        FileOutputStream fileOutputStream = null;
        byte[] bs = null;
        try {
            int len = list.size();
            for (int i = 0; i < len; i++) {
                newUrl = new URL(list.get(i));
                hconnection = (HttpURLConnection) newUrl.openConnection(); //打开连接
                inputStream = hconnection.getInputStream();  //获取流
                bs = getBytesFromInputStream(inputStream); //流转btye[]
                filepath = filepath + list.get(i).substring(list.get(i).lastIndexOf("/") + 1); //获取图片名称
                System.out.println("生成图片路径:" + filepath);
                fileOutputStream = new FileOutputStream(new File(filepath));
                fileOutputStream.write(bs); //写出
            }
        } catch (Exception e) {
            e.printStackTrace();
        } finally {
            try {
                inputStream.close();
                fileOutputStream.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
    }

    /**
     * @param @param  inputStream
     * @param @return byte[]
     * @return byte[]    返回类型
     * @throws
     * @Title: getBytesFromInputStream
     * @Description: InputStream流转换byte[]
     */
    public static byte[] getBytesFromInputStream(InputStream inputStream) {
        byte[] bs = null;
        try {
            byte[] buffer = new byte[1024];
            int len = 0;
            ByteArrayOutputStream arrayOutputStream = new ByteArrayOutputStream(); //
            while ((len = inputStream.read(buffer)) != -1) {
                arrayOutputStream.write(buffer, 0, len);
            }
            bs = arrayOutputStream.toByteArray();
        } catch (IOException e) {
            e.printStackTrace();
        }
        return bs;
    }


}